001 /*
002 * LSFResourceStrategy.java
003 *
004 * Created on April 1, 2003, 3:30 PM
005 *
006 * This file is part of the STAR Scheduler.
007 * Copyright (c) 2002-2003 STAR Collaboration - Brookhaven National Laboratory
008 *
009 * STAR Scheduler is free software; you can redistribute it and/or modify
010 * it under the terms of the GNU General Public License as published by
011 * the Free Software Foundation; either version 2 of the License, or
012 * (at your option) any later version.
013 *
014 * STAR Scheduler is distributed in the hope that it will be useful,
015 * but WITHOUT ANY WARRANTY; without even the implied warranty of
016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017 * GNU General Public License for more details.
018 *
019 * You should have received a copy of the GNU General Public License
020 * along with STAR Scheduler; if not, write to the Free Software
021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022 */
023 package gov.bnl.star.offline.scheduler.lsf;
024
025 import gov.bnl.star.offline.scheduler.*;
026 import gov.bnl.star.offline.scheduler.catalog.PhysicalFile;
027
028 import java.util.Hashtable;
029 import java.util.Iterator;
030 import java.util.Set;
031 import java.util.logging.Logger;
032 import java.util.regex.Matcher;
033 import java.util.regex.Pattern;
034
035
036 /** Encapsulate the resource usage parameter (-R) for an LSF farm. Essentially,
037 * its a placeholder for the prepareResourceUsageSwitch() which will calculate the
038 * appriate string. The administrator should be able to create different
039 * subclasses for each farm as needed.
040 * <p>
041 * This class provides a scheleton for a typical resource usage for files residing
042 * on different NFS servers. It was designed to fit STAR needs at two sites. In case
043 * it doesn't fit the needs, the class can be subclassed to define new behaviour.
044 * <p>
045 * The resource switch is prepared in this way:
046 * Takes the list of input files and iterates over them.<br>
047 * Checks if each file needs a resource (<CODE>resourceRequired()</CODE>).<br>
048 * To determine that, checks if the file is on NFS, and looks for the typical data
049 * vault prefix (<CODE>getDataVaultPrefix</CODE>). For example, "/star/dataxx/...";
050 * it extract the vaultNumber and it substitute it in the resourceSyntax <br>
051 * Increases the count of the resource (<CODE>increaseResourceUsage</CODE>)<br>
052 * Prepares the switch (<CODE>prepareSwitch</CODE>)
053 * <p>
054 * The resource value for each resource will the be given by the following formula: <br>
055 * <code>min (base+nCount*fileFactor, max)</code>
056 * <p>
057 * The subclass need to implement the abstract methods, which defines things that
058 * always change between different farms.
059 * <p>
060 * TODO Further work should be done to be able to configure the ResourceStrategy
061 * depending on the farm on which the job will be dispatched. This should be
062 * included in the GRID dispatchers.
063 **
064 * @author Gabriele Carcassi, Jerome Lauret
065 * @version July 24, 2003
066 */
067 public class LSFResourceStrategy {
068 static private Logger log = Logger.getLogger(LSFResourceStrategy.class.getName());
069
070 /** Base value for the resource value formula. */
071 protected int base = 0;
072
073 /** Returns the base value for the resource value formula. */
074 public int getBase() {
075 return base;
076 }
077
078 /** Changes the base value for the resource value formula. */
079 public void setBase(int base) {
080 this.base = base;
081 }
082
083 /** FileFactor value for the resource value formula. */
084 protected int fileFactor = 2;
085
086 /** Returns the fileFactor value for the resource value formula. */
087 public int getFileFactor() {
088 return fileFactor;
089 }
090
091 /** Changes the fileFactor value for the resource value formula. */
092 public void setFileFactor(int fileFactor) {
093 this.fileFactor = fileFactor;
094 }
095
096 /** Max parameter for the standard increaseResourceUsage implementation. */
097 protected int max = 50;
098
099 /** Returns the max value for the resource value formula. */
100 public int getMax() {
101 return max;
102 }
103
104 /** Changes the max value for the resource value formula. */
105 public void setMax(int max) {
106 this.max = max;
107 }
108
109 private Hashtable resources;
110
111 private String dataVaultPrefix;
112
113 /** Returns the prefix with which a file on NFS identifies on which file server or
114 * data vault is residing. For example, "/star/data07/muDST/..." represents a file
115 * on data vault "07" (that is the data vault number or id), therefore the prefix is "/star/data".
116 * @return The prefix that identifies data vaults on NFS.
117 */
118 public String getDataVaultPrefix() {
119 return dataVaultPrefix;
120 }
121
122 /** Changes the prefix of a NFS file. The resource identifies the data vault number
123 * by looking at a prefix in the file name. For example "/star/data07/minbias/..."
124 * would have "/star/data" as a prefix.
125 */
126 public void setDataVaultPrefix(String dataVaultPrefix) {
127 this.dataVaultPrefix = dataVaultPrefix;
128 }
129
130 private String resourceNameSyntax;
131
132 /** Returns the syntax to be used to convert a data vault number to the LSF resource.
133 */
134 public String getResourceNameSyntax() {
135 return resourceNameSyntax;
136 }
137
138 /** Changes the syntax to be used to convert a data vault number to the LSF resource.
139 * It can be any string, and it should contain the $vaultNumber variable, which will
140 * be substituted with the particular vault number. For example, to have something
141 * like "dv7io", you should have "dv$vaultNumberio".
142 */
143 public void setResourceNameSyntax(String resourceNameSyntax) {
144 this.resourceNameSyntax = resourceNameSyntax;
145 }
146
147 private boolean trimVaultNumberLeadingZeros;
148
149 /** Returns true if the leading zeros of a vault number will be removed when
150 * creating the corresponding LSF resource.
151 */
152 public boolean isTrimVaultNumberLeadingZeros() {
153 return trimVaultNumberLeadingZeros;
154 }
155
156 /** Set to true if the leading zeros of a vault number will be removed when
157 * creating the corresponding LSF resource. Basically, 07 will be changed to 7.
158 */
159 public void setTrimVaultNumberLeadingZeros(boolean trimVaultNumberLeadingZeros) {
160 this.trimVaultNumberLeadingZeros = trimVaultNumberLeadingZeros;
161 }
162
163 /** Given the vault id, returns the LSF resource name to be used in the -R
164 * parameter.
165 * @param vaultNumber the vault id, as identified in <CODE>getDataVaultPrefix</CODE>
166 * @return the LSF resource name for the vault
167 */
168 protected String prepareResourceName(String vaultNumber) {
169 if ((removeCharactersFromVaultNumber) && (!vaultNumber.matches("[a-zA-Z]+"))) {
170 Pattern pattern = Pattern.compile("[^a-zA-Z]+");
171 Matcher matcher = pattern.matcher(vaultNumber);
172 StringBuffer buf = new StringBuffer(vaultNumber.length());
173 StringBuffer vaultNumberBuf = new StringBuffer(vaultNumber);
174 while (matcher.find()) {
175 buf.append(vaultNumberBuf.subSequence(matcher.start(), matcher.end()));
176 }
177 vaultNumber = buf.toString();
178 }
179
180 if (trimVaultNumberLeadingZeros) {
181 while (vaultNumber.startsWith("0") && (vaultNumber.length() != 1)) {
182 vaultNumber = vaultNumber.substring(1);
183 }
184 }
185
186 if (trimVaultNumberDecimal) {
187 int dec = vaultNumber.indexOf('.');
188 if (dec != -1) vaultNumber = vaultNumber.substring(0, dec);
189 }
190
191 String name = resourceNameSyntax.replaceAll("\\$vaultNumber", vaultNumber);
192
193 return name;
194 }
195
196 private String switchSyntax;
197
198 /** Holds value of property trimVaultNumberDecimal. */
199 private boolean trimVaultNumberDecimal;
200
201 /** Holds value of property removeCharactersFromVaultNumber. */
202 private boolean removeCharactersFromVaultNumber;
203
204 /** Returns the syntax that will be used to prepare the final switch
205 */
206 public String getSwitchSyntax() {
207 return switchSyntax;
208 }
209
210 /** Changes the syntax that will be used to prepare the final switch.
211 * You can use the $definedNames, $nameEqualValueCommaSeparated
212 * or $nameEqualValueColumnSeparated variables in your switch.
213 * For example, the following switch:
214 * "\"select[$definedNames] rusage[$nameEqualValueCommaSeparated]\""
215 * will produce something like: "select[defined(01)] rusage[01=50]"
216 */
217 public void setSwitchSyntax(String switchSyntax) {
218 this.switchSyntax = switchSyntax;
219 }
220
221 /** Prepares the -R parameters to give to LSF at the end of the resource
222 * distribution. It will use <CODE>definedNames()</CODE>,
223 * <CODE>nameEqualValueCommaSeparated()</CODE> or
224 * <CODE>nameEqualValueColumnSeparated()</CODE> to prepare it more easily.
225 * @return the -R parameter for the current job
226 */
227 protected String prepareSwitch() {
228 if (determineResourceNotUsed()) {
229 return null;
230 }
231
232 String fullSwitch = switchSyntax.replaceAll("\\$definedNames", definedNames());
233 fullSwitch = fullSwitch.replaceAll("\\$nameEqualValueCommaSeparated", nameEqualValueCommaSeparated());
234 fullSwitch = fullSwitch.replaceAll("\\$nameEqualValueColumnSeparated", nameEqualValueColumnSeparated());
235
236 return fullSwitch;
237 }
238
239 /** Increase the resource usage as part of the standard implementation. The standard
240 * formula starts counting from <CODE>base</CODE>, adds <CODE>fileFactor</CODE> for
241 * each usage until <CODE>max</CODE> is reached. In pseudo-code:
242 * <CODE>if (nRes == 0) nRes = base;
243 * nRes += nRes;
244 * if (nRes > max) nRes = max;
245 * </CODE>
246 * <p>
247 * One can modify the three protected parameters (base, fileFactor, max) to tune
248 * the usage.
249 * @param resource the resource for which to increase the counting.
250 */
251 protected void increaseResourceUsage(String resource) {
252 increaseResourceUptoLimit(resource, base, fileFactor, max);
253 }
254
255 /** Addes a new resource, and initializes the value to <CODE>base</CODE>.
256 * @param resName the name of the resource
257 * @param base the initial value for the resource
258 */
259 protected void addResource(String resName, int base) {
260 if (!resources.containsKey(resName)) {
261 resources.put(resName, new Integer(base));
262 } else {
263 throw new RuntimeException("Can't add resource " + resName +
264 " because it's already present.");
265 }
266 }
267
268 /** Increases the value associated to the given resource by a given amount. If the
269 * resource doesn't exists, first it will be created and initialized.
270 * @param resName the name of the resource
271 * @param base the initial value if the resource has to be created
272 * @param amount the amount to increase the resource by
273 * @return the new value associated to the resource
274 */
275 protected int increaseResource(String resName, int base, int amount) {
276 if (!resources.containsKey(resName)) {
277 addResource(resName, base);
278 }
279
280 int returnValue = prepareResourceValue(resName) + amount;
281 Integer newValue = new Integer(returnValue);
282
283 resources.remove(resName);
284 resources.put(resName, newValue);
285
286 return returnValue;
287 }
288
289 /** Increases the value associated to the given resource by a given amount. If the
290 * resource doesn't exists, first it will be created and initialized. If the
291 * increased amount is over the limit, the value will be set to the limit.
292 * @return the new value associated to the resource
293 * @param limit upper limit to which the resource value will be set
294 * @param resName the name of the resource
295 * @param base the initial value if the resource has to be created
296 * @param amount the amount to increase the resource by
297 */
298 protected int increaseResourceUptoLimit(String resName, int base,
299 int amount, int limit) {
300 if (!resources.containsKey(resName)) {
301 addResource(resName, base);
302 }
303
304 int returnValue = prepareResourceValue(resName) + amount;
305
306 if (returnValue > limit) {
307 return limit;
308 }
309
310 Integer newValue = new Integer(returnValue);
311
312 resources.remove(resName);
313 resources.put(resName, newValue);
314
315 return returnValue;
316 }
317
318 /** Returns the value currently associated to the resource.
319 * @param resName resource name
320 * @return resource value
321 */
322 protected int prepareResourceValue(String resName) {
323 return ((Integer) resources.get(resName)).intValue();
324 }
325
326 /** Returns the -R parameter to be used to dispatch the given job.
327 * @param job the job to be dispatched
328 * @return the -R parameter to use
329 */
330 public String prepareResourceUsageSwitch(Job job) {
331 initResources();
332 calculateLSFResources(job);
333
334 return prepareSwitch();
335 }
336
337 /** Initializes the resource table. */
338 protected void initResources() {
339 resources = new Hashtable();
340 }
341
342 /** Returns a string already formatted containing names and values comma separated
343 * (i.e. "res1=34,res2=12").
344 * @return resource name/value pairs comma separated (i.e. "res1=34,res2=12")
345 */
346 protected String nameEqualValueCommaSeparated() {
347 String rusage = "";
348 Set resNames = resources.keySet();
349 Iterator iter = resNames.iterator();
350 boolean first = true;
351
352 while (iter.hasNext()) {
353 if (!first) {
354 rusage += ",";
355 }
356
357 Object key = iter.next();
358 rusage += (key + "=" + resources.get(key));
359 first = false;
360 }
361
362 return rusage;
363 }
364
365 /** Returns a string already formatted containing names and values column
366 * separated (i.e. "res1=34:res2=12").
367 * @return resource name/value pairs column separated (i.e. "res1=34:res2=12")
368 */
369
370 protected String nameEqualValueColumnSeparated() {
371 String rusage = "";
372 Set resNames = resources.keySet();
373 Iterator iter = resNames.iterator();
374 boolean first = true;
375
376 while (iter.hasNext()) {
377 if (!first) {
378 rusage += ":";
379 }
380
381 Object key = iter.next();
382 rusage += (key + "=" + resources.get(key));
383 first = false;
384 }
385
386 return rusage;
387 }
388
389
390
391 /** A list of names, comma separated, of all the resources used.
392 * @return (i.e. "res1,res2")
393 */
394 protected String definedNames() {
395 String defined = "";
396 Set resNames = resources.keySet();
397 Iterator iter = resNames.iterator();
398 boolean first = true;
399
400 while (iter.hasNext()) {
401 if (!first) {
402 defined += " && ";
403 }
404
405 Object key = iter.next();
406 defined += ("defined(" + key + ")");
407 first = false;
408 }
409
410 return defined;
411 }
412
413 /** Filles the resource table with all the resources. It basically iterates over the
414 * input files and calls <CODE>increaseResourceUsage</CODE> for each file that map
415 * to a resource.
416 * @param job the job
417 */
418 protected void calculateLSFResources(Job job) {
419 log.fine("Calculating LSF resources");
420 for (int i = 0; i < job.getInput().size(); i++) {
421 PhysicalFile file = (PhysicalFile) job.getInput().get(i);
422 String res = resourceRequired(file);
423
424 if (res != null) {
425 increaseResourceUsage(res);
426 }
427 }
428 }
429
430 /** Determines which resource the job might need to access the file.
431 * @param file input file the job will be using
432 * @return the resource needed
433 */
434 protected String resourceRequired(PhysicalFile file) {
435 if ("NFS".equals(file.getStorage())) {
436 log.finest("Found NFS file: " + file);
437
438 String path = file.getPath();
439
440 if (path.startsWith(getDataVaultPrefix())) {
441 int bar = path.indexOf('/', getDataVaultPrefix().length());
442 String serverNumber = path.substring(getDataVaultPrefix()
443 .length(), bar);
444
445 return prepareResourceName(serverNumber);
446 }
447 }
448
449 return null;
450 }
451
452 /** Determines whether any resources at all have been assigned.
453 * @return true if any resource is used.
454 */
455 protected boolean determineResourceNotUsed() {
456 return resources.isEmpty();
457 }
458
459 /** Getter for property trimVaultNumberDecimal.
460 * @return Value of property trimVaultNumberDecimal.
461 *
462 */
463 public boolean isTrimVaultNumberDecimal() {
464 return this.trimVaultNumberDecimal;
465 }
466
467 /** Setter for property trimVaultNumberDecimal.
468 * @param trimVaultNumberDecimal New value of property trimVaultNumberDecimal.
469 *
470 */
471 public void setTrimVaultNumberDecimal(boolean trimVaultNumberDecimal) {
472 this.trimVaultNumberDecimal = trimVaultNumberDecimal;
473 }
474
475 /** If true removes all the characters from the vault number.
476 * @return Value of property removeCharactersFromVaultNumber.
477 *
478 */
479 public boolean isRemoveCharactersFromVaultNumber() {
480 return this.removeCharactersFromVaultNumber;
481 }
482
483 /** Set to true removes all the characters from the vault number
484 * @param removeCharactersFromVaultNumber New value of property removeCharactersFromVaultNumber.
485 *
486 */
487 public void setRemoveCharactersFromVaultNumber(boolean removeCharactersFromVaultNumber) {
488 this.removeCharactersFromVaultNumber = removeCharactersFromVaultNumber;
489 }
490
491 }